'use strict'

var initDb = require('./utils/initDb')
// initDb();

var article = require('./models/articles');

var Crawler = require("crawler");

var fs = require("fs");

var request = require('request');

var c = new Crawler({
    rateLimit:200 ,
    
    maxConnections: 10,
    // This will be called for each crawled page
    callback: function (error, res, done) {
        if (error) {
            console.log(error);
        } else {
            // var $ = res.$;

            // var list = $('#content .article table').toArray();
            // console.log(list);
            // list.forEach(element => {
            //     var txt = $(element).find('.pl2 a ').text().trim().replace(/[ \r\n]/g, '');
            //     article.create({ title: txt })
            //     console.log(txt);
            // });

            var $ = res.$;

            var list = $('#content .article table').toArray();
            console.log(list);
            list.forEach(element => {
                var txt = $(element).find(' img').attr('src');
                var imgName = txt.split('/').slice(7,8).toString();
                
                console.log(txt);
                console.log(imgName);
                // require(txt).pipe(fs.createWriteStream('./images/'  + imgName + '.jpg'))
                request(txt).pipe(fs.createWriteStream('./images/'  + imgName + '.jpg'))
                article.create({title:txt});

            });



        };
        done();
    }
});

for (var i = 0; i <= 250; i = i + 25) {
    
    c.queue('https://book.douban.com/top250?start=' + i);
}

